#!/usr/bin/env python
#-*- coding: utf-8 -*-
#
# BSD License
# Copyright (c) 2011, Wang Qiang
# All rights reserved.

"""
Retrieval news according to config 
"""
import os
import fnmatch
from Config import Config
from WebReader import WebReader
from twisted.internet import reactor
from twisted.internet import defer
from twisted.python import log
from twisted.python.logfile import DailyLogFile


def stop_reactor(_):
    """
    Stop the reactor
    """
    reactor.stop()


def taskConfigStep(args, dir, files):
    """
    Config Task in a dir.
    Used by os.path.walk
    @param args: retain the deferred list and config file
    """
    for file in files:
        if fnmatch.fnmatch(file, "*.xml"):
            reader = WebReader(args[1], os.path.join(dir, file))
            d = reader.openLink()
            args[0].append(d)

def taskConfig(root, config):
    """
    Config task according to the xml files in root dir
    @param root: root dir
    @return : deferred list
    """
    deferlist = []
    os.path.walk(root, taskConfigStep, (deferlist, config))
    dl = defer.DeferredList(deferlist)
    dl.addCallback(stop_reactor)
    return dl

def NewsRetrieval():
    """
    Retrieval news.
    """
    # Load config
    config = Config('./config/config.xml')
    config.load()
    log.startLogging(DailyLogFile.fromFullPath(config.log))
    # Config the task
    dl = taskConfig('./webs', config)
    reactor.run()


if __name__ == "__main__":
    NewsRetrieval()
